package engine

import (
    "crawler/fetcher"
    "log"
    "time"
)

var rateLimit = time.Tick(time.Millisecond * 20)

func Work(request Request) ParseResult {
    var result ParseResult
    log.Printf("fetch url:%s", request.Url)
    <-rateLimit
    data, err := fetcher.FetchData(request.Url)
    if err != nil {
        return result
    }
    
    return request.Parser.Parse(data)
}
